{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2422b17b8ef27a54",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-18T02:43:05.639293Z",
     "start_time": "2025-07-18T02:43:01.336148Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dadudu/miniconda3/envs/mini-gpt/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading Model from https://www.modelscope.cn to directory: /Users/dadudu/.cache/modelscope/hub/models/openai-community/gpt2\n",
      "Downloading Model from https://www.modelscope.cn to directory: /Users/dadudu/.cache/modelscope/hub/models/openai-community/gpt2\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from modelscope import AutoTokenizer, AutoModelForCausalLM\n",
    "\n",
    "model_name = \"openai-community/gpt2\"\n",
    "gpt_model = AutoModelForCausalLM.from_pretrained(model_name)\n",
    "gpt_tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "# gpt_tokenizer.pad_token = gpt_tokenizer.eos_token\n",
    "gpt_tokenizer.add_special_tokens({'pad_token': '[PAD]'})\n",
    "\n",
    "# gpt_tokenizer.add_special_tokens({'mask_token': '[MASK]'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8fc9e4f98354c4ea",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-18T02:41:35.923754Z",
     "start_time": "2025-07-18T02:41:33.191195Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-07-18 10:55:35,454 - modelscope - WARNING - Use trust_remote_code=True. Will invoke codes from chinese-poetry-collection. Please make sure that you can trust the external codes.\n",
      "2025-07-18 10:55:37,195 - modelscope - WARNING - Reusing dataset dataset_builder (/Users/dadudu/.cache/modelscope/hub/datasets/modelscope/chinese-poetry-collection/master/data_files)\n",
      "2025-07-18 10:55:37,196 - modelscope - INFO - Generating dataset dataset_builder (/Users/dadudu/.cache/modelscope/hub/datasets/modelscope/chinese-poetry-collection/master/data_files)\n",
      "2025-07-18 10:55:37,196 - modelscope - INFO - Reusing cached meta-data file: /Users/dadudu/.cache/modelscope/hub/datasets/modelscope/chinese-poetry-collection/master/data_files/7c9a7977d937face2055b6145eaf516f\n"
     ]
    }
   ],
   "source": [
    "from modelscope.msdatasets import MsDataset\n",
    "\n",
    "dataset = MsDataset.load('modelscope/chinese-poetry-collection', subset_name='default', split='train')\n",
    "data = dataset.to_hf_dataset().select(range(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fe1dd619a0366028",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-18T02:41:37.253362Z",
     "start_time": "2025-07-18T02:41:37.228036Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Map: 100%|██████████| 1/1 [00:00<00:00, 76.52 examples/s]\n"
     ]
    }
   ],
   "source": [
    "# 数据预处理\n",
    "def tokenize_function(examples):\n",
    "    # 拼接EOS标记并编码\n",
    "    poems = [p + gpt_tokenizer.eos_token for p in examples[\"text1\"]]\n",
    "    return gpt_tokenizer(poems, max_length=128, padding=\"max_length\", truncation=True)\n",
    "\n",
    "\n",
    "tokenized_datasets = data.map(tokenize_function, batched=True, remove_columns=data.column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bf48383f531d54ea",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-18T02:44:00.106382Z",
     "start_time": "2025-07-18T02:44:00.103053Z"
    }
   },
   "outputs": [],
   "source": [
    "from transformers import DataCollatorForLanguageModeling\n",
    "\n",
    "# 创建数据收集器\n",
    "data_collator = DataCollatorForLanguageModeling(\n",
    "    tokenizer=gpt_tokenizer,\n",
    "    mlm=False  # 使用CLM（因果语言模型）\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d99c2772ec4bef3e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-18T02:44:42.541439Z",
     "start_time": "2025-07-18T02:44:42.537863Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[    1,     2,     3,     4],\n",
       "         [    1,     2, 50257, 50257]]),\n",
       " 'labels': tensor([[   1,    2,    3,    4],\n",
       "         [   1,    2, -100, -100]])}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_collator([[1, 2, 3, 4], [1, 2]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2eb65b25b022f95f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-18T02:50:04.024582Z",
     "start_time": "2025-07-18T02:49:54.346697Z"
    },
    "jupyter": {
     "is_executing": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dadudu/miniconda3/envs/mini-gpt/lib/python3.10/site-packages/torch/utils/data/dataloader.py:683: UserWarning: 'pin_memory' argument is set as true but not supported on MPS now, then device pinned memory won't be used.\n",
      "  warnings.warn(warn_msg)\n",
      "`loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='100' max='100' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [100/100 00:15, Epoch 100/100]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>2.666500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>0.653400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>0.105700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>0.064100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>0.021700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60</td>\n",
       "      <td>0.063600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70</td>\n",
       "      <td>0.043900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80</td>\n",
       "      <td>0.070200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>90</td>\n",
       "      <td>0.032700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>0.056300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import Trainer, TrainingArguments\n",
    "\n",
    "# 训练配置\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./huggingface_gpt\",\n",
    "    per_device_train_batch_size=4,\n",
    "    num_train_epochs=100,\n",
    "    # eval_strategy=\"epoch\",\n",
    "    # save_strategy=\"epoch\",\n",
    "    logging_steps=10\n",
    ")\n",
    "\n",
    "# 创建Trainer\n",
    "trainer = Trainer(\n",
    "    model=gpt_model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_datasets,\n",
    "    # eval_dataset=tokenized_datasets,\n",
    "    data_collator=data_collator\n",
    ")\n",
    "\n",
    "# 开始训练\n",
    "trainer.train()\n",
    "trainer.save_model(\"./huggingface_gpt/model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fed9769a193d5a54",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-17T13:54:28.893939Z",
     "start_time": "2025-07-17T13:54:23.514544Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "半生长以客为家，罢直初来瀚海槎。始信人间行不尽，天涯更复有天涯。\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"mps\")\n",
    "\n",
    "\n",
    "def generate(sentence, max_length=100):\n",
    "    inputs = gpt_tokenizer.encode(sentence, return_tensors='pt')\n",
    "    input_ids = inputs.to(device)\n",
    "    output = gpt_model.generate(\n",
    "        input_ids,\n",
    "        max_length=max_length,\n",
    "        pad_token_id=gpt_tokenizer.pad_token_id,\n",
    "        eos_token_id=gpt_tokenizer.eos_token_id,\n",
    "        do_sample=True\n",
    "    )\n",
    "    return gpt_tokenizer.decode(output[0], skip_special_tokens=True)\n",
    "\n",
    "\n",
    "print(generate(\"半生\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a0afe2894a4d2671",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-17T13:55:26.594325Z",
     "start_time": "2025-07-17T13:55:22.539325Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Device set to use mps:0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "半生长以客为家，罢直初来瀚海槎。始信人间行不尽，天涯更复有天涯。\n"
     ]
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "# 使用pipeline进行文本生成\n",
    "text_generator = pipeline(\n",
    "    task=\"text-generation\",\n",
    "    model=\"./huggingface_gpt/model\",\n",
    "    tokenizer=gpt_tokenizer\n",
    ")\n",
    "\n",
    "# 生成文本\n",
    "results = text_generator(\n",
    "    \"半生\",\n",
    "    pad_token_id=gpt_tokenizer.pad_token_id,\n",
    "    eos_token_id=gpt_tokenizer.eos_token_id,\n",
    "    do_sample=True\n",
    ")\n",
    "\n",
    "# 输出结果\n",
    "print(results[0]['generated_text'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
